"""
steam游戏数据爬取
天津工业大学 大数据实训 第四组
杨玺锟
2020-1-8
"""
# -*- coding: utf-8 -*-
import datetime
import re

import scrapy

from ..items import SteamSpiderItem


class SteamSpider(scrapy.Spider):
    name = 'steam'
    allowed_domains = ['store.steampowered.com']
    start_urls = []
    base_url = 'https://store.steampowered.com/search/?os=win&filter=topsellers&page={}'

    # 循环翻页，生成热销榜url
    for page in range(0, 10):  # 控制页数
        start_urls.append(base_url.format(page))

    def parse(self, response, null=None):
        game_review_count = 0
        positive_review_ratio = 0
        games_list = response.xpath('//*[@id="search_resultsRows"]/a')  # 这里的a标签数组要去掉，加数组指的是一个游戏

        for game in games_list:
            print()
            game_review_count = 0
            positive_review_ratio = 0

            # 设置默认item值
            item = SteamSpiderItem()
            item['game_name'] = ''
            item['game_price'] = ''
            item['price_discount'] = ''
            item['release_date'] = ''
            item['game_review_count'] = ''
            item['positive_review_ratio'] = ''
            item['game_type'] = ''

            game_link = game.xpath('./@href').extract()[0]  # 游戏详情页url

            try:
                game_name = game.xpath('./div[2]/div[1]/span/text()').extract()[0]
                print("游戏名：%s" % game_name)
                game_price_str = game.xpath('./div[2]/div[4]/div[2]/text()').extract()[0].strip()  # strip()删除多余空格
                # 看游戏是否打折
                if not game_price_str:
                    game_price_str = game.xpath('./div[2]/div[4]/div[2]/text()').extract()[1].strip().lstrip('¥ ')
                    game_price_str = re.findall(r'\b\d+\b', game_price_str.replace(',', ''))  # 个别游戏价格会过千
                    # findall结果是list，所以下面结果用game_price_str[0]
                    game_price = int(game_price_str[0])
                    price_discount_str = game.xpath('./div[2]/div[4]/div[1]/span/text()').extract()[0].lstrip(
                        '-').rstrip(
                        '%')  # 折扣转换int
                    price_discount = int(price_discount_str) / 100
                else:
                    game_price_str = re.findall(r'\b\d+\b', game_price_str.replace(',', ''))  # 个别游戏价格会过千
                    # findall结果是list，所以下面结果用game_price_str[0]
                    game_price_str = game_price_str[0].lstrip('¥ ')
                    game_price = int(game_price_str)
                    price_discount = 0
                print("游戏价格：%d" % game_price)
                print("游戏折扣：%d" % price_discount)
                release_date = game.xpath('./div[2]/div[2]/text()').extract()  # 可能为空的结果用list表示，因为空结果没有extract()[0]
                if release_date:
                    try:
                        release_date = datetime.datetime.strptime(release_date[0], "%d %b, %Y").strftime("%Y%m%d")
                        print("发行日期：%s" % release_date)
                    except Exception as e1:
                        print(e1)
                        release_date = null
                    print("发行日期：%s" % release_date)
                else:
                    release_date = null
                    print("发行日期：%s" % release_date)
                review = game.xpath(
                    './div[2]/div[3]/span/@data-tooltip-html').extract()  # 同上可能为空的结果用list表示
                # 格式：特别好评<br>29,572 篇用户的游戏评测中有 86% 为好评。
                # print(review)
                if review:
                    split = re.findall(r'\b\d+\b', review[0].replace(',', ''))  # 提取测评数以及好评比 ->['29752', '86']
                    if split:
                        game_review_count = int(split[1])  # 提取测评数转为int
                        print("评论数：%d" % game_review_count)
                        positive_review_ratio = int(split[0]) / 100  # 提取好评百分比为int
                        print("好评比：%d" % positive_review_ratio)
                else:
                    game_review_count = 0
                    positive_review_ratio = 0
                    print("无评论")

                item['game_name'] = game_name
                item['game_price'] = game_price
                item['price_discount'] = price_discount
                item['release_date'] = release_date
                item['game_review_count'] = game_review_count
                item['positive_review_ratio'] = positive_review_ratio
            except Exception as e:
                print(e)
                print('game_link', game_link)
                continue

            detail_request = scrapy.Request(
                url=game_link,
                callback=self.game_details,
                headers={
                    "Accept-Language": "zh-CN,zh;q=0.9",
                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                                  "Chrome/72.0.3626.121 Safari/537.36 "
                },
                meta={"item": item, 'game_link': game_link},
            )
            yield detail_request
            # break

    def game_details(self, response):
        item = response.meta["item"]
        # game_link = response.meta["game_link"]
        try:
            game_type = response.xpath('//*[@id="game_highlights"]/div[1]/div/div[4]/div/div[2]/a[1]/text()').extract()
            if not game_type:  # 捆绑包
                game_type = response.xpath('/html/body/div[1]/div[7]/div[4]/div[1]/div[2]/div/div[3]/div/div[2]/div['
                                           '1]/p/a[1]/text()').extract()
                if not game_type:
                    game_type = ''
                    print("无类型")
                else:
                    game_type = game_type[0]
            else:
                game_type = game_type[0].strip()
            print("游戏类型：%s" % game_type)
            item['game_type'] = game_type

            game_about = response.xpath('//*[@id="game_highlights"]/div[1]/div/div[2]/text()').extract()
            if game_about:
                game_about = game_about[0].strip()
            else:  # 捆绑包/需要验证
                game_about = ''
            item['game_about'] = game_about

            img_src = response.xpath('//img[@class="game_header_image_full"]/@src').extract()
            if img_src:
                img_src = img_src[0]
            else:
                img_src = response.xpath('//img[@class="package_header"]/@src').extract()[0]
            item['img_src'] = img_src
        except Exception as e1:
            print(e1)
        yield item
